<?php

/**
 * 抓取一个文章 信息
 */
namespace console\controllers;

use backend\models\AntiSpamAnnotation;
use backend\models\AssenssmentLog;
use backend\models\Category;
use backend\models\DaliyNews;
use backend\models\KeywordTask;
use backend\models\Task;
use backend\models\TopicNews;
use common\models\Onearticle;
use Think\Exception;
use yii\console\Controller;
use Yii;
use yii\db\ActiveRecord;
use backend\models\Video;
use yii\helpers\Html;

class OneController extends Controller
{
    /**
     * 批处理脚本的实现
     * 一次取50
     */
    public function actionArticle()
    {
        $min = 0;
        $max = 2500;
        $batchnum = 50;
        $j = 0;
        for ($id = $min; $id <= $max + $batchnum; $id += $batchnum) {
            $url = [];
            for ($i = $id; $i < $id + $batchnum; $i++) {
                $url[] = 'http://v3.wufazhuce.com:8000/api/essay/' . $i;
            }
            $datas = static::multiRequest($url);
            file_put_contents("$id.one", $datas);
            foreach ($datas as $data) {
                $res = json_decode($data, true);
                if (isset($res['res']) && $res['res'] == 0) {
                    echo $j++ . "done" . PHP_EOL;
                    $ret = $res['data'];
                    $one = new Onearticle();
                    $one->content_id = $ret['content_id'];
                    $one->title = $ret['hp_title'];
                    $one->content = $ret['hp_content'];
                    $one->author = $ret['hp_author'];
                    $one->author_it = $ret['auth_it'];
                    $one->guide_word = $ret['guide_word'];
                    $one->wb_img_url = $ret['wb_img_url'];
                    $one->save(false);
                }
            }
        }
    }

    /**
     * @param $urls
     * @return array
     */
    public static function multiRequest($urls)
    {
        $mh = curl_multi_init();
        $curl_array = array();
        foreach ($urls as $i => $url) {
            $curl_array[$i] = curl_init($url);
            curl_setopt($curl_array[$i], CURLOPT_RETURNTRANSFER, true);
            curl_setopt($curl_array[$i], CURLOPT_URL, $url);
            curl_setopt($curl_array[$i], CURLOPT_HEADER, false);
            curl_setopt($curl_array[$i], CURLOPT_RETURNTRANSFER, true);
            curl_setopt($curl_array[$i], CURLOPT_SSL_VERIFYPEER, false);
            curl_multi_add_handle($mh, $curl_array[$i]);
        }
        $active = null;
        do {
            curl_multi_exec($mh, $active);
        } while ($active > 0);
        $response = [];
        foreach ($urls as $i => $url) {
            $response[$url] = curl_multi_getcontent($curl_array[$i]);
        }
        foreach ($urls as $i => $url) {
            curl_multi_remove_handle($mh, $curl_array[$i]);
        }
        curl_multi_close($mh);
        return $response;
    }

    public function singleCurl($url){
        $header = array(
            'User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:53.0) Gecko/20100101 Firefox/53.0',
            'Accept-Language:zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
            'Content-Type:application/x-www-form-urlencoded;charset=utf-8'
        );
        $curl = curl_init();
        curl_setopt($curl, CURLOPT_URL, $url);
        curl_setopt($curl, CURLOPT_HEADER, 1);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($curl, CURLOPT_HTTPHEADER, $header);

        $data = curl_exec($curl);
        curl_close($curl);
        return $data;
    }

    /**
     * ./yii /one/abtest 20170522
     * @param string $date
     */
    public function actionAbtest($date = ""){
        //查找是否存在实验数据
        $date = $date ? date("Y-m-d",strtotime($date)) : date("Y-m-d",strtotime("-1 day"));
        $list = DaliyNews::selectAbtestType($date);

        foreach ($list as $value){
            $this->actionNews('all', $value['date'], $value['country'], $value['ty']);
        }
    }

    /**
     * ./yii /one/news 20170522 us all
     * 将远程数据加入redis
     * @param string $date
     * @param string $country
     * @param string $ty
     */
    public function actionNews($date = "", $country = "", $ty = "")
    {
        error_reporting(0);

        $default = array('us','in','hi','id','af');
        $date = $date ? date("Y-m-d",strtotime($date)) : date("Y-m-d",strtotime("-1 day"));
        $ymd = str_replace("-","",$date);
        $ty = $ty ? $ty : 'all';

        if( $country && !in_array($country,$default) ){
            echo "国家编码不存在".PHP_EOL;
            return false;
        }

        $ctrs = array();
        if( $country ){
            $ctrs[] = $country;
        }else{
            $ctrs = $default;
        }

        foreach ( $ctrs as $country ) {
            try {
                $impression = DaliyNews::selectRemoteData($date, $country, $ty);
            } catch (\yii\db\Exception $exception) {
                $impression = DaliyNews::selectRemoteData($date, $country, $ty);
            }

            //等待5分钟后重试
            while (!$impression){
                $impression = DaliyNews::selectRemoteData($date, $country, $ty);
                sleep(300);
            }

            echo "start set redis... ".$country."-".$ty."-".$ymd.PHP_EOL;
            $this->addRedis($impression);
            echo "end set redis... ".$country."-".$ty."-".$ymd.PHP_EOL;
        }
    }

    /**
     * 批量将网页内容存到redis中
     * @param $impression
     */
    public function addRedis($impression){
        $redis = Yii::$app->redis;
        $operaDetailUrl = array(
            'us' => 'http://news.opera-api.com/news/detail/',
            'in' => 'http://news-in.op-mobile.opera.com/news/detail/',
            'hi' => 'http://news-in.op-mobile.opera.com/news/detail/',
            'id' => 'http://news-sg.op-mobile.opera.com/news/detail/',
            'af' => 'http://news-af.op-mobile.opera.com/news/detail/',
        );

        $max = count($impression);
        for ($i = 0; $i <= $max; $i++) {
            if( $impression[$i]['news_id'] ){
                $oneurl = $operaDetailUrl[$impression[$i]['country']] . $impression[$i]['news_id'];

                $data = @file_get_contents($oneurl);
                $redis->setex($impression[$i]['news_id'],36000*24*5,$data);
                echo "{$i} {$impression[$i]['country']} {$impression[$i]['date']} {$oneurl}".PHP_EOL;
            }
        }
    }

    /**
     * ./yii /one/generator 20170522 us en
     * 生成任务,插入相应新闻信息
     */
    public function actionGenerator($ymd = "",$country = "",$lang = "")
    {
        //error_reporting(0);

        $date = $ymd ? date("Y-m-d",strtotime($ymd)) : date("Y-m-d",strtotime("-1 day"));
        $ymd = $ymd ? $ymd : date('Ymd',strtotime("-1 day"));
        $country = $country ? $country : "";
        $ctrs = array('us','in','id','af','su');
        $language = array('in' => array('en','hi'),'su' => array('ru'));

        if( $country ){
            $this->generatorNews($date,$ymd,$country,$lang);
        }else{
            foreach ($ctrs as $country){
                //增加language
                if( isset($language[$country]) ){
                    foreach ( $language[$country] as $lang ){
                        $this->generatorNews($date,$ymd,$country,$lang);
                    }
                }else{
                    $this->generatorNews($date,$ymd,$country);
                }

            }
        }
        echo "完成".PHP_EOL;
    }

    private function generatorNews($date,$ymd,$country,$language = ""){
        $impression = DaliyNews::selectRemoteData($date, $country, "all",$language);
        $temp = array();
        foreach ( $impression as $new ){
            $temp[$new['news_id']] = $new;
        }
        $data = array();
        foreach ( $temp as $value ){
            $data[] = $value;
        }
        $count = count($data);

        if( $count ){
            $taskModel = new Task();
            $ret = $taskModel->createOne(Task::TASK_TYPE_ALL,$count,$country,$ymd,"all",0,$language);

            if( $ret['id'] ){
                $ty = "all";
                echo "start ".$country."-".$ty."-".$ymd.PHP_EOL;
                DaliyNews::initDailyNews( $impression, $country, $ty, $ymd, $ret['id'] );
                echo "end ".$country."-".$ty."-".$ymd.PHP_EOL;

                $this->actionDisgustingNews($ret['id'],"all",Task::TASK_TYPE_DISGUSTING_NEWS,$language);
            }
        }
    }

    /**
     * ./yii /one/generator-abtest 20170522 us
     * 生成实验数据任务
     */
    public function actionGeneratorAbtest($ymd = "",$country = "",$lang = "")
    {
        error_reporting(0);
        //默认设置
        $ymd = date("Ymd",strtotime("-1 day"));
        $country = 'in';
        $language = 'hi';

        $date = $ymd ? date("Y-m-d",strtotime($ymd)) : date("Y-m-d",strtotime("-1 day"));
        $ymd = $ymd ? $ymd : date('Ymd',strtotime("-1 day"));
        $ctrs = array('us','in','id','af');

        $country = in_array($country,$ctrs) ? $country : "";

        $err = array();
        $list = DaliyNews::selectAbtestType( $date, $country, $language);

        foreach ($list as $value){
            $impression = DaliyNews::selectRemoteData($value['date'], $value['country'], $value['ty'], $language);

            $count = count($impression);
            if( $count ){
                $taskModel = new Task();
                $ret = $taskModel->createOne(Task::TASK_TYPE_ABTEST,$count,$country,$ymd,$value['ty'],0,$language);

                if( $ret['id'] ){
                    $ty = $value['ty'];
                    echo "start ".$country."-".$ty."-".$ymd.PHP_EOL;
                    DaliyNews::initDailyNews( $impression, $country, $value['ty'], $ymd, $ret['id'] );
                    echo "end ".$country."-".$ty."-".$ymd.PHP_EOL;

                    //$this->actionDisgustingNews($ret['id'],$value['ty'],Task::TASK_TYPE_ABTEST_DISGUSTING);
                }
            }
        }

        echo "完成".PHP_EOL;

    }

    /**
     * impression top600
     * @param $taskid
     */
    public function actionDisgustingNews( $taskid, $ty = 'all',$type = Task::TASK_TYPE_DISGUSTING_NEWS,$lang = "" ){
        $list = DaliyNews::find()->select('entryid,newsid,country,ty,ymd,score,impression,taskid,status,ctime,mtime')
            ->where(array('taskid' => $taskid,'ty' => $ty))
            ->orderBy("impression desc")->limit(600)->asArray()->all();

        $taskModel = new Task();
        $newsTask = $taskModel::findOne($taskid);
        $task = $taskModel->createOne($type,count($list),$newsTask->country,$newsTask->ymd,$newsTask->ty,0,$lang);

        if( $task['id'] ){
            foreach ( $list as &$value ){
                $value['taskid'] = $task['id'];
            }
            Yii::$app->db->createCommand()
                ->batchInsert('remark_disgusting_news',
                    ['entryid','newsid', 'country', 'ty','ymd', 'score','impression','taskid','status','ctime','mtime'],
                    $list
                )
                ->execute();
        }
    }

    /**
     * ./yii /one/insert-video file 3 banfuguang
     * 插入视频数据
     */
    public function actionInsertVideo($file,$userid,$username,$country){
        error_reporting(0);
        $taskModel = new Task();
        $ymd = date("Ymd",time());
        $ret = $taskModel->createOne(Task::TASK_TYPE_VIDEO,0,$country,$ymd,"");

        if( !$ret['ret'] ) return false;

        $taskid = $ret['id'];
        $line_count = 0;
        if(file_exists($file)){
            $fp = fopen($file,"r");

            while(!feof($fp)){//循环读取，直至读取完整个文件
                $line_count++;

                $str = fgets($fp);
                if( $str ){
                    $data = json_decode($str,true);
                    $data['taskid'] = $taskid;
                    $data['userid'] = $userid;
                    $data['username'] = $username;
                    $id = Video::insertData($data);
                }
            }

            $taskModel = Task::findOne($taskid);
            $taskModel->total = $line_count;
            $taskModel->save();
        }
    }

    /**
     * ./yii /one/insert-video file 3 banfuguang
     * 插入视频数据
     */
    public function actionInsertCategoryData( $file, $country ){
        error_reporting(0);
        $taskModel = new Task();
        $ymd = date("Ymd",time());
        $ret = $taskModel->createOne(Task::TASK_TYPE_CATEGORY,0,$country,$ymd,"");

        if( !$ret['ret'] ) return false;

        $taskid = $ret['id'];

        $line_count = 0;
        if(file_exists($file)){
            $fp = fopen($file,"r");

            while(!feof($fp)){//循环读取，直至读取完整个文件
                $line_count++;

                $str = fgets($fp);
                if( $str ){
                    $data = json_decode($str,true);

                    $data['taskid'] = $taskid;
                    $id = Category::insertData($data);
                }
            }

            $taskModel = Task::findOne($taskid);
            $taskModel->total = $line_count;
            $taskModel->save();
        }
    }

    /**
     * ./yii /one/insert-topic-data file 3 banfuguang
     * 插入topic数据
     */
    public function actionInsertTopicData( $file, $country, $topic ){
        error_reporting(0);
        $taskModel = new Task();
        $ymd = date("Ymd",time());
        $ret = $taskModel->createOne(Task::TASK_TYPE_TOPIC,0,$country,$ymd,"",$topic);

        if( !$ret['ret'] ) return false;

        $taskid = $ret['id'];

        $line_count = 0;
        if(file_exists($file)){
            $fp = fopen($file,"r");

            while(!feof($fp)){//循环读取，直至读取完整个文件
                $line_count++;

                $str = fgets($fp);
                if( $str ){
                    $data = json_decode($str,true);
                    $data['taskid'] = $taskid;
                    $id = TopicNews::insertData($data,$topic);
                }
            }

            $taskModel = Task::findOne($taskid);
            $taskModel->total = $line_count;
            $taskModel->save();
        }
    }

    /**
     * 关键词样本
     * ./yii one/import-keywords-json
     */
    public function actionImportKeywordsJson($file,$country){
        //$file = "/home/sdev/id_evaluation_300_0809.json";
        //$file = "/usr/local/var/www/test/id_evaluation_3_0830.json";

        error_reporting(0);
        if(file_exists($file)) {
            $fp = file_get_contents($file);
            $data = json_decode($fp,true);

            if( $data ){
                $ctime = time();
                $date = date("Ymd");

                $task = KeywordTask::find()->where(array('type' => 1))->orderBy('id desc')->asArray()->one();
                $title_array = explode("-",$task['title']);
                $no = 1;
                if( $title_array['2'] == $date ){
                    $no = $title_array['3'] + 1;
                }
                //$country = "";
                $title = "test-keywords-{$date}-{$no}";
                $tasksql = "INSERT INTO `keyword_task` (`id`, `title`, `country`, `ymd`, `description`, `type`, `ctime`) ".
                    "VALUES (NULL, '{$title}', '{$country}', {$date}, '', '1',{$ctime})";
                Yii::$app->db->createCommand($tasksql)->execute();

                foreach ($data as $key => $value){
                    //document
                    $task = KeywordTask::find()->where(array('type' => 1))->orderBy('id desc')->asArray()->one();
                    $taskid = $task['id'];
                    $docid = $key;
                    $title = html::encode($value['title']);
                    $content = html::encode($value['content']);

                    $url = html::encode($value['url']);
                    $category = "";
                    $status = 0;
                    $ctime = time();
                    $mtime = time();
                    $sql = "insert into keyword_documents (`taskid`, `docid`, `title`, `content`, `url`, `category`, `status`, `ctime`, `mtime`) ".
                        "VALUE ('{$taskid}','{$docid}','{$title}','{$content}','{$url}','{$category}','{$status}',{$ctime},{$mtime})";

                    Yii::$app->db->createCommand($sql)->execute();

                    //keyword
                    if( isset($value['supervised_keywords']) ){
                        foreach ($value['supervised_keywords'] as $sort => $val){
                            $ctime = time();
                            $mtime = time();

                            foreach ($val as $keyword => $score){
                                $keyword = addslashes($keyword);
                                $sql = "insert into keyword_keywords (`taskid`, `docid`, `keyword`, `impression`, `sort`, `type`, `ctime`, `mtime`) ".
                                    "VALUE ('{$taskid}','{$docid}','{$keyword}','{$score}','{$sort}',1,{$ctime},{$mtime})";
                            }

                            Yii::$app->db->createCommand($sql)->execute();
                        }
                    }

                    //keyword1
                    if( isset($value['keywords1']) ){
                        foreach ($value['keywords1'] as $sort => $val){
                            $ctime = time();
                            $mtime = time();

                            foreach ($val as $keyword => $score){
                                $keyword = addslashes($keyword);
                                $sql = "insert into keyword_keywords (`taskid`, `docid`, `keyword`, `impression`, `sort`, `type`, `ctime`, `mtime`) ".
                                    "VALUE ('{$taskid}','{$docid}','{$keyword}','{$score}','{$sort}',2,{$ctime},{$mtime})";
                            }

                            Yii::$app->db->createCommand($sql)->execute();
                        }
                    }
                    //keyword2
                    if( isset($value['keywords2']) ){
                        foreach ($value['keywords2'] as $sort => $val){
                            $ctime = time();
                            $mtime = time();

                            foreach ($val as $keyword => $score){
                                $keyword = addslashes($keyword);
                                $sql = "insert into keyword_keywords (`taskid`, `docid`, `keyword`, `impression`, `sort`, `type`, `ctime`, `mtime`) ".
                                    "VALUE ('{$taskid}','{$docid}','{$keyword}','{$score}','{$sort}',3,{$ctime},{$mtime})";
                            }

                            Yii::$app->db->createCommand($sql)->execute();
                        }
                    }

                }
            }

        }
    }

    /**
     * 修复数据
     * @param $ymd
     */
    public function actionFix($ymd){
        $list = AssenssmentLog::find()->where(array('ymd' => $ymd))->orderBy("id desc")->asArray()->all();
        foreach ( $list as $value ){
            if( $value['taskid'] && $value['newsid'] ){
                DaliyNews::updateAll(
                    array('textlabel' => $value['textlabel'],'imagelabel' => $value['imagelabel'],'label' => $value['label'],'status' => 1,'userid' => $value['userid'],'username' => $value['username']),
                    array('taskid' => $value['taskid'],'newsid' => $value['newsid'],'status' => 0,'ymd' => $ymd,'country' => ['us','in'])
                );
            }

        }
    }
}
